#coding=utf-8
'''
采集 http://www.biedoul.com/wenzi/
'''
import sys
reload(sys)
sys.setdefaultencoding( 'utf-8')
import urllib, urllib2, re, time
import MySQLdb


class Spider:
    def __init__(self):
        # self.siteURL = 'http://www.biedoul.com/pic/gif/'
        self.siteURL = 'http://www.biedoul.com/wenzi/'

    def getPage(self,pageIndex):
        url = self.siteURL + str(pageIndex) + '/'
        print url
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        #print response.read()
        return response.read()


    def getContents(self, pageIndex):
        page = self.getPage(pageIndex)
        #print page
        #pattern = re.compile('<div.*?class=dzlist rmargin10">.*?<div.*?.*?<div.*?.*?<div.*?<a href=(.?)',re.S)
        pattern = re.compile('<div.*?class="dzlist-.*?>.*?<div.*?</div>.*?<div.*?<a.*?>(.*?)</a>.*?<div.*?</div>.*?</div>.*?<div.*?>.*?<a.*?<p.*?>(.*?)</p>.*?<p>(.*?)</p>.*?</div>.*?<div.*?>.*?<a.*?>(.*?)</a>.*?<a.*?>(.*?)</a>.*?<a.*?>(.*?)</a>.*?</div><div.*?<div.*?<a.*?>(.*?)</a>.*?<a.*?>(.*?)</a>.*?</div>',re.S)
        items = re.findall(pattern, page)
        
        
        for item in items:
            # 作者   标题  内容   tag1 tag2 tag3 
            #print  item[1].decode('utf-8'), item[2].decode('utf-8')
            #如何删除指定的字符串。'<br />'
            #item2 = item[2].replace('<br />', '')
            #print item[0],item[1],item[2].replace('<br />', '')
           
            d0 = item[0].decode('utf-8')   # 作者
            d1 = item[1].decode('utf-8')  # 标题
            d2 = item[2].decode('utf-8')  # 内容
            d3 = item[3].decode('utf-8')   # tag1
            d4 = item[4].decode('utf-8')   # tag2 
            d5 = item[5].decode('utf-8')   # tag3
            d6 = item[6].decode('utf-8')   # zanUp
            d7 = item[7].decode('utf-8')   # zandown
            #print d6,d7
            print  d1 + '  ||||', d2 + '  ||||', d3 + '  ||||', d4 + '  ||||', d5
            '''
            #保存到sql
            '''
            # 打开数据库连接
            db = MySQLdb.connect("localhost","root","root","mycms" , charset="utf8")
            # 使用cursor()方法获取操作游标 
            cursor = db.cursor()
            # 查看是否有  已有  tittle
            sql_SELECT_article_id = "SELECT id FROM myapp_article WHERE title='%s'" %(d1)
            cursor.execute(sql_SELECT_article_id)
            article_id1 = cursor.fetchall()
            print article_id1
            # 如果已有 article_id1 不添加，没有才添加
            if not article_id1:
                #SQL 插入语句  表myapp_article
                now_time = time.strftime('%Y-%m-%d %H:%M:%S')
                sql = "INSERT INTO `mycms`.`myapp_article` (`title`, `content`, `creation_date`, `update_time`, `published`, \
                                        `zanUp_num`, `zanDown_num`, `comment_num`, `keep_num`, `author_id`, `column_id`,`tag1`,`tag2`,`tag3`) \
                                            VALUES ('%s','%s','%s','%s','1','%s','%s','0','0','1','1','%s','%s','%s');" %(d1,d2,now_time,now_time,d6,d7,d3,d4,d5)
                
                try:
                    cursor.execute(sql)
                    db.commit()
                except:
                    print u'提交失败'
                    print u'continue 语句跳出本次循环'
                    continue
                # 关闭数据库连接
                db.close()                       
            else:
                print '这个article 已存在'
                # 关闭数据库连接
                db.close()      
 
spider = Spider()
#spider.getContents(11)

for i in range(100, 105):
    spider.getContents(i)
    print(i)
    time.sleep(3)
else:
    print('for循环结束')
